{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shallow Neural Network in Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build a shallow neural network to classify MNIST digits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/the-deep-learners/deep-learning-illustrated/blob/master/notebooks/shallow_net_in_keras.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import SGD\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_valid, y_valid) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, 1, 9, 2, 1, 3, 1, 4, 3, 5], dtype=uint8)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[0:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAE8CAYAAAD+LWvAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHpFJREFUeJzt3WeYlNUVwPE/YMFesRfAEiyxoGJPFFTsisaGqJTYeyUmBgsYiQZR7AXEFhuJyWN51FhiRbGAsSKKoqgoWFAUUEo+rGdfZgss7Mzcmdn/78s+uzuzc7k7nD3vfc89t9ns2bORJBVf89QDkKSmygAsSYkYgCUpEQOwJCViAJakRAzAkpSIAViSEjEAS1IiBmBJSmShIr9epW67a5Z6ADi3heTcFlaTnV8zYElKxAAsSYkYgCUpEQOwJCViAJakRAzAkpRIscvQVEE++eQTAK688koABg4cCMDpp58OwKmnngrAmmuumWB0UukzA5akRJoV+Uiigr3YrFmzAJg+fXqd37/11lsB+OGHHwB4++23AbjiiisA+OMf/8jVV18NwGKLLQbAgAEDADj++OPn9fKlUNBetF/kp59+CsCmm24KwLffflvn45ZbbjkAJk6c2JiXa1JzO7/eeecddtllFwBGjRoFQKtWrRr69FKYWyih+b3pppsAOO6444CquDJ69GgA1l9//fn9cW7EkKRSVTZrwJMnTwZg5syZALz++usAPPbYY0CWhd14440N+nmtW7cG4MwzzwRg8ODBLLPMMgDsuOOOAHTs2DEPI68c48aNA2CnnXYC4JtvvgGgWbOqP/Qxf4suuigAX375JQBjx45l7bXXBqBFixZFG2+xjBkzBsjmo0OHDkV77ZdeeolOnToV7fUq1RNPPAHAGWecAUDz5lluGu/vQij5ADx+/HgANttsMyB7ky+omNjBgwcD2XJDr169WGmllQBYcsklgfm6lKtIP//8M5AF3t133x3Ibr7VFL+jiy++GIAddtgBgPXWW6/6D2OvXr0KN+BE4j/vu+++CxQnAMfS4ZgxY3jvvfcK/nqVLuZw2rRpRX1dlyAkKZGSz4BXWGEFAFZeeWWg4RnwbrvtlvP8f/7zn0B2eRyX0arf2WefDVB9c3Jenn76aSC70dmlSxegau5HjhxZgBGWhkGDBgHZe64YpkyZAsAll1xSXe7X1K/YFkTcjL/gggtyvt6+fXugaolziSWWKNjrmwFLUiIlnwHHGu3QoUMBGDZsGADbbrstAAceeGDO42Pd8d///jcAiyyyCAATJkwAsk0Dql+s8d5xxx1Att4YIrONue/WrRuQbbjYYIMNAOjduzdQ9TsrcrljUcWN4WKKMinI5lsN9/777wOw5557AvD111/nfL9///5AdmO5UMyAJSmRks+Aw1ZbbQXAJptsAmSZ7TnnnAPApZdeCkDfvn1zvh9WWWUVoGrNTHWLDRabb745kJX2RRnO4YcfDmTF6rF+Fp8feuihACy++OIArLbaakBV5cntt98OwB/+8AegMrYnf/bZZ0A2b8U0Z8a26667Fv31y93NN98M1K7oOeCAAwDYeeedizIOM2BJSqRsMuAQVQwhtruGuCMdmykKWURdKSZNmgTAX//6VyCrNInKkzZt2gDZluy4uoi63/g4Nz/++CMAl112GZD9nspZbAKKf1sxRIXJG2+8Uf21qPTRvNV8H8a+gJjDuIIuFjNgSUqk7DLgmk477TQARowYAcD9998PwFtvvQXAxhtvnGZgZWDGjBkAnHXWWUBW9RB3fh999FEA1l13XSDbGdcYH374YaN/Rql48803cz5vyJVAY/3pT38CsvXnTTbZpNb9DtUW9zP222+/Or8fdcDt2rUr1pAAM2BJSqbsM+D46x+9BmJffvyl23///QHYfvvtgayG1bVh+Pjjj4Es8w0vvvgiULv9XtRkq25bb7113n5WtFV99dVXgez9fc899+Q8btCgQbRs2TJvr1upnn32WQBeeOGFnK8fdNBBAHTv3r3YQwLMgCUpmbLPgMPyyy8PZOuW0bkrGq7HxyFDhgDZLq7ofNYUnXjiiUC20y2uDhag8fRczZo1q/pucyXviKuvMT1ka7ZxcED0zYg18Z9++gmAq666Csh210UfgugzEdlurMe7C27uXn75ZQCOOuqonK/vs88+QFbDnuoqwgxYkhKpmAw4RC/WqIKIAyLvu+8+AHr27AnABx98AGQdv5ZaaqmijjO1kSNH8swzzwDZenish+Vb8+bNq19jyy23LMhrpBA7/uLftu+++wLwq1/9qtZjhw8fDmRXAAstVPVfL67AYv04KlKijj0qKyITjh2EUQ9sB7S6xdXINttsU+f3o7KnkJ3OGsIMWJISqZhDOesTHe7jzn4cYBj/7t/97ndA7bvL86kUSirma26HDx9enWVFz4bo7dDYdfGoL47dbmeffXZ1dn3bbbcBtXt1zEXJz20c+Prf//53nj+oa9euQJaBxS7DeXn44YcB2HvvvYGsXjV+ZwuoFOYWChAX/vznPwNZV7OaYk2+wFcQHsopSaWq4taAa4q7m3ECRhwKGVnav/71LwBGjx5d59pdUxBzlK/M97rrrgOyTnWtW7eu3sFVibu24g57zTvt+fTggw/mfB73MpQrOtNF3/CaevToAZTO2rkZsCQlUrEZcKzxxFlwcRc6srQQfYbzXftaTo444ohGPT+yjuimdu211wJZthG1lsqf6FurXFFlEx3+QufOnYGGn29YLGbAkpRIxWTAEydOBOCaa64B4JZbbgFg/PjxdT4+1oJbt24NNL3eELNnz66uBInz9uLOcUPdddddAJx88slA1kf4lFNOAWDgwIH5GKrUYF9++SWQ9fkNcT5hqd2DMAOWpETKNgOeMmUKAA888AAAF110EQDvvffeXJ/XsWNHIKsP3GKLLQo1xJLWrFmz6qw/rhJiDnv16gVkuwNjV+ENN9wAZJ2lPvroIwDWWWcdIDsTLjJg5V9ctYwbNw6Atm3bphxOyYgdhNFro6Y4S7LUlE0Ajq2XcYheHIU+cuTIuT4vmphceOGFQHbTraktOcxNNH6JADx48GAga3A05/E3c9pjjz2ArPHRSSedVNBxKnvf1hdompqaZWex9BBHl51//vlA+i3H9XEJQpISKekMeOrUqdVHDj333HMAvPvuu3N9zp577glAnz59gKyZycILL1yoYZaljTbaqHpb9uOPP57zvViSqHnc+korrQRkh3PO70075c+TTz4JQKdOnRKPJK1Yiqz5Xo2b63HzrVSZAUtSIiWVAcdNnb/85S9AVWYWNxvqEy0B4zjpE044ASi9cpNSs/TSS1evm0WDnPpunvXr1w+Ao48+GvAY9JQquaF9U2QGLEmJlFQG/I9//API7sLPqX379gAcdthhQNbQ+phjjgHSHSlSzqL5Tlw1xEeVnjhC6/rrr088ktKy+uqrA7DXXnsBWVlquTADlqREKr4he5GUQlGxc1s4zm1hNdn5NQOWpEQMwJKUiAFYkhIxAEtSIgZgSUqk2FUQkqRfmAFLUiIGYElKxAAsSYkYgCUpEQOwJCViAJakRAzAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKREDsCQlYgCWpEQMwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSkRA7AkJWIAlqREDMCSlIgBWJISMQBLUiIGYElKxAAsSYkYgCUpEQOwJCViAJakRAzAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKREDsCQlYgCWpEQMwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSkRA7AkJWIAlqREDMCSlIgBWJISMQBLUiIGYElKxAAsSYkYgCUpEQOwJCViAJakRAzAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKZGFivx6s4v8esXSLPUAcG4LybktrCY7v2bAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKZFil6FJmkPfvn0B6NOnDwAdOnQA4LHHHgNgmWWWSTMwFYUZsCQl0mz27KLWQDfZgusiKPrcTp8+HYCff/4ZgOeeew6ATz/9FICjjjoKgIUWatSFVkXO7bfffgvAeuutB8DXX38NQLNmVf/ckSNHAvDrX/863y89p1KYWyjA/E6aNAmAGTNmADBixAgA9ttvPwCaN29Y7tmjRw9uuOEGAFq0aDG/w3AjhiSVKteA1WCRtQ0YMACAJ598EoCXXnqpzsdHJhzrm8osvvjiAOy7774ADB06NOFoyt+ECRMAuO222wC48cYbAZg1axYAH3/8MZBlvnGlMS9Dhw5lueWWA6Bfv34ALLroonkadQUtQXz00UdA9kZ+5JFHAHj55ZdzHnfnnXcCsOaaawLwn//8B4Du3bvTunXrBX35UriUy/vcTpw4EYArr7wy5+PUqVOrXvCX906bNm0AWGGFFQB49dVXAVh55ZUBGDVqFK1atVrQYVTk3Ib4T33++ecDLkEsqO7duwNwxx13zP2FfnnPNjQAz2n06NEArLPOOg19iksQklSqyn4J4vnnnwfg4IMPBuCLL74Asr90BxxwAACffPIJAN26dct5fjxu4sSJXHPNNYUfcAmbNm0akGVl1113HQCTJ0+u8/GRnT399NNAdsMjMt/4XUyePLkxGXBFirmOTFeNs88++wC1M+DVVlsNgLPOOgvIliRq3oR79tlnAbj//vsLOs6azIAlKZGyy4DjL1is+e61114ATJkyBYD9998fyLK4KPOZOXMmAD179gTg7rvvzvm52223XQFHXR7iaqJ///5zfdyGG24IwDPPPAPA0ksvDcBXX31VwNFVlijde/vtt+v8/osvvgjAWmutBbghY166dOkCZOV8ITLdJZdccq7PP/bYYwHYYIMNgOymHWQxY+21187PYOccX95/oiSpQcouA37qqacA6Ny5c87XDznkEACGDBkC1C4ViU0CNTPfqHyIv6BNWX2lUOuvvz4AHTt2BODiiy8Gssw3jBs3rnCDqzBLLbUUAKeffjoAxx9/fM734/OoLIl7GapbZLo135MN9dprrwHZBo45xVVIIzcU1ckMWJISKZsMeNCgQUCWMUQdXxT59+7dG6i/SPq0006r8+v33HMPkBXGN2XXXnstANtuuy0Au+++O5BVNSyxxBJzff6XX35ZwNFVpmOOOQaonQGrOOLKOGrcf/zxx1qPOfvsswv2+mbAkpRIyWfA119/PZBlvpHhHnrooQCce+65ACy88MI5z4ua1Ndffx2AMWPGAFndb2TUW265ZcHGXm5iXfKEE05YoOfH1mTNv/rqU5VfUblz5plnAvDWW28B8NNPP9X5+B133LGgvxN/25KUSElnwNOmTatuWB1rvpH5RrVDTVEHGFURUTURot7v6KOPzv+AK9ywYcMA+O6774Da++qjB0SIGu22bdsWa4hla36bxChXNIq69957AXj44YfrfNwDDzwA1D/Pyy67LJA19dlhhx1qXV3nkxmwJCVS0hnwzJkzq/sJhIEDBwLwww8/AFlWFtUMw4cPB7IsLf7Sxcff//73ACyyyCKFHHpZi11an332GZBVmtTcZ1/fumV0mrvlllvq/L6UL59//jkAO+20EwAffPBBo35e9JTYc889G/VzGsr/GZKUSElnwC1atGCVVVYBsobLyy+/PFD/Gk7sWom1nOiCFrWs7du3L9yAy1T0yRg/fjyQZRMxd1EjHZntHnvsAcBdd90FZH04QlSgPPTQQwB07dp1QY5zkRos7kfMq7/5vKpNYu331FNPBWCzzTbL1xDrZAYsSYmUdAbcsmXL6p0q22yzDZCd0hAduY444ggAjjzySCDbrRVfjyzOnUa1ReY7atQoALbeeuuc78fOuE6dOgHZSQBxIsb//vc/oPaRRHG10qNHD6CqCiJ+diH201eC+jKzOLHFXhB1W3XVVYHs5Jv77rsPgN122w2Y972ewYMHA9mJJMVmBixJiVTMmXAhdrxFB6/IKKI+8MADDyzEy5ZC8eZ8ze3MmTOr97+fc845Od/r2rUrkB1s2LJlSyDbJ7/33nsD2UkYsTvxsssuA7KMOqogIDuxJCoqavZnXWONNeobatnN7YKINfL67m3EAadxLyNPSmFuoQjzW584maTm+/GVV14BGr0G7JlwklSqKm5BLv6i1dxZFHfum7pYa7ziiiuqO8hFD4joBxy9liPzjT6/sXsw9tPHmXDRY7ldu3YATJ8+HYCTTz4ZqNq1eOuttwLZlUiIXXLvvfdefv6BZeq8884Dsl7LNd100005j1N+RB/gVMyAJSmRisuAIytT3R588EGgqn9yrHvF/vgtttgCgNGjRwNZJ7rYARfVD1dffTWQrRXXPIUg1oQ32WQToCrbjrX3yORC7Gxs6mKuNG8zZ87kjTfeAGCjjTYCandDnJeoLjnooIPyO7j5ZAYsSYlUXBVE/GWMu5exBhy9IQp08kUp3E1u0NxGtcGECROq13gj8508eTIAb775Zp3Pve666wDo1asXULQeD2Uzt/kQV3A1T0uOtfs4eTp2hDZSKcwtNHB+o8LpggsuqO79Et0P53UWXFy9jRgxAsjqquM9HyI+xOPivsYCmuf8VtwSxNixY1MPoaTFIaQTJkyovmEZx9GHbt26AbDrrrsC2Q3M2N5tc53C6dChAwDvvPNOztedc+jevTuQu/EnlrDmFYBjmS1KJ2uW+0VAjkbtjQy8DeZvVZISqbgliGhPt9pqqwFZ5vD9998DLkFEidjw4cOrM9/YzhlN7GNpokQa6JTN3OZDHKEVy0LVA/jl/2lsxW+KSxDbb789UHvr+3y90C/zuPrqqwNZy4ILL7wQyPtWeTdiSFKpqrgMOMTNjFhLiwX8Nm3aFOLlSiGTSLads8Ca1NzGTaFoJhPHPJkBZ+1SBw0axOWXX96gHxxNu2KNOOY1NhXF1V+BmAFLUqmq2Az4iSeeALJttV26dAGyTQQV2NTEDLhwnNvCmq/5nTFjBo888giQHTE2adIkAHr27AnAvvvuC2SHC9RstlMkZsCSVKoqNgOOu/3RFDyawMTaT7RizNPhnKWQSZilFY5zW1hNdn7NgCUpkYrNgENkwv379wegb9++QN4bXJdCJtFks4gicG4Lq8nOrxmwJCVS8RlwkZRCJuHcFo5zW1hNdn7NgCUpkWJnwJKkX5gBS1IiBmBJSsQALEmJGIAlKREDsCQlYgCWpEQMwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSkRA7AkJWIAlqREDMCSlIgBWJISMQBLUiIGYElKxAAsSYkYgCUpEQOwJCViAJakRAzAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKREDsCQlYgCWpEQMwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSkRA7AkJWIAlqREDMCSlIgBWJISMQBLUiIGYElKxAAsSYkYgCUpEQOwJCViAJakRAzAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKREDsCQlYgCWpEQMwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSmRhYr8erOL/HrF0iz1AHBuC8m5LawmO79mwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSkRA7AkJVLsOmA1cQcddBAAs2dXlX4OGzYs5XCS+eKLLwB49NFHAejfvz8AHTt2BKBDhw45jz/88MMBaNGiRbGGqCIwA5akRComA545cyYAH3zwAQCnnXYaAA8//HCyMSlz8cUXA/DQQw8BcPrpp6ccTjIPPvggAF27dgXg+++/z/n+O++8A8A111yT8/XIiNu1a1foIaqIzIAlKZGKyYCnT58OZBnCGmusAcCUKVMAWHLJJdMMTAwYMKA6A15kkUUA2GuvvVIOKZlOnToB2fuxZgZcn+233x6Ap59+GoCNN964AKNTsVVMAK5p/PjxAEyePBkwAKf03HPP8dNPPwGwzz77ALDddtulHFIyiy22GAA33HADAIcddhgAP/zwAwBt27YFYOzYsTnP+/rrrwF44IEHAANwsUyePLn6vXvvvfcC0K9fv5zHxA3Sv/3tb/P9812CkKREKjYDjjInNd6YMWMA6NOnDwBDhgwBsmyuPs8++ywAL7zwAhtuuCEAAwcOLNQwy0pcCWy66aZA1RwBrLjiikDtDDgcd9xxRRhd0/X2228DcPfddwNVN0O/+eYbAJo1q7u75BNPPLHAr2cGLEmJNCtypliwF/vxxx+B2mu977//PpCtrRVIKTS2LtjcbrbZZgC88cYbAIwePRqAddddd67P22qrrQB45ZVXeOmll4DaGwwaoKLn9sUXXwTgrLPOAuD555+f6+NjA8dKK62Uj5cvhbmFhA3Ze/fuDcBrr70G1J3NLrPMMgCcfPLJAOy4444A7LzzzgAstFC9Cwk2ZJekUlWxa8Bh1KhRQMEz4Iq29NJLA9kaWNwVrs+nn34KZGvHzZs3ry4TVK5tttkGgEceeQSAXXbZBaD6iqGm8847D4Abb7yxCKOrPFOnTgXgoosuAuCyyy4DoFWrVgDstNNOAFxyySVAVdyI0snIhPPJDFiSEqmYDLh586q/JcsttxxA9Z3L2Nqp+XfVVVcBMHz4cAA233xzAFq3bl3n4yMzjuwhNsF07ty5ydb9zsszzzwDZBnviBEj5vr42MihBTNgwAAALr30UgAuvPBCIFsLjmy3WMyAJSmRismAW7ZsCWT1lbfddlvK4ZS17777DshaJC688MIA3HnnnQAsvvjidT4vsonrr78egLXWWguwIdKcJk6cCMBuu+0GwJtvvgnAjBkzGvT8eJ7m7ueffwaytfJBgwYB8Pe//x2A3XffHcgqfOZSyVBQZsCSlEjFZMBqvM8//xzI7sRHzWlktuuvv36dz4vMuOZe+Mg6lPnwww8BePfdd4GGZ74h5vT888/P78AqzNVXXw1k9dXHH388kO08TJXx1mQGLEmJlMafgQKaNGlS6iGUrFmzZgHw1FNPAdn6Ynw9KkuiBeIqq6wCwFFHHQXAtGnTABg6dCiQ9d+IZut77713QcdfjmIn4O233w7AkUceCWT1qfMSNdaauzPOOAPIatd79OgBlE7mG8yAJSmRiukFEbp37w5kVRDLLrsskPVTLZBS2FM/33MbmW3N2tJ4T2y00UZA1iEqxMGRsdPtk08+AbIMOXox50lZzm1Dvf7660BWeRLiiK0uXboA8O233wJw9NFHA3nbCVcKcwsFmN9dd90VgCeffBKAtddeG8j6Kcd7u8DsBSFJpariMuDo4xmHHpoB1+3555+v3vcedb7LL788AI8//jgASy21FJAdcHr//ffnvuAv751YZ4uPcRzUq6++mvNzF1DZzW1eXvCXub322msBOOmkkwDYYIMNgGx3YiP7E5TC3EIj5vejjz4CYM011wSgRYsWQLamfssttwBZJ7PoaxId/fLUVa4+ZsCSVKpK65ZgHrRp0ybn8+hPEGfDFaKjUTkaOHBgdT/fqC2NdbOaoqYysoro3FVTZG37778/0OjMt0mLNeDIfMOiiy4K1H86Q1MwZcqU6kNdI5O95557APjtb38LZKe1xD2hyIBjrT36lBQ4A54nM2BJSqTiMuBYAwqRlcXecFU55JBD6Ny5M5Cti9UnsoZYdwxx5ts666yT8/VYd9eCu/zyy+v8euzsmtfvrJK1a9euuiokqp0i863p5ptvzvn84IMPBmD11Vcv4AgbzgxYkhKpuCqI0L59eyA7ESNOEohO+HlWCgtyeZ/b2OkWXdH69u0LUH3CcZwRV2BlN7dTp06t7j3Qs2dPAH7zm9806LmxNhl39SPTC1HNE32vG6kU5hbmc36HDBnCKaecAmRnQda08cYbA1m3ubjfEWe+xfwWmFUQklSqKm4NOBxwwAFA1n2qT58+KYdTlqJ3ar9+/QBYddVVgXmf3NvU9e7dm1tvvRXIrsDuvfdeAFZccUUgqxCJXYRRz3ruuecCtTPfuAqJ2uymrGfPntXVIHGSyLBhw3IeE32Xu3XrBmQnYaywwgrFGmaDVOwSRASNuJnx1VdfAQUr3ymFS7m8zW2U7MWBke+//z4AV1xxBQAnnnhivl6qIcpubseOHVs9RzVL9tZbbz0Att56ayDbGhtzHuJ9Gg3D4/j6PB+ZUwpzCwmPpS8wlyAkqVRV7BJEiEu5OOwwMg/Vb4cddgCyZjunnnoqUPTMt2y1bdu2uiwqbsbtt99+QDan8bE+can82muvFWqYKgFmwJKUSMWuAceBkNGQfdy4cQC0atWqEC9XCmtpeZvbwYMHA3DssccC2U23RFcPZT23ceTQXXfdlfP1uCKLbd4hysuiTWWBy6VKYW7BNWBJUrFVbAYc65WxhhZ3owvUjKcUMokmm0UUgXNbWE12fs2AJSmRis2Ai6wUMgnntnCc28JqsvNrBixJiRiAJSkRA7AkJWIAlqREDMCSlEixqyAkSb8wA5akRAzAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKREDsCQlYgCWpEQMwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSkRA7AkJWIAlqREDMCSlIgBWJISMQBLUiIGYElKxAAsSYkYgCUpkf8DkVo09UXlLuYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7ccf591e80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5,5))\n",
    "for k in range(12):\n",
    "    plt.subplot(3, 4, k+1)\n",
    "    plt.imshow(X_train[k], cmap='Greys')\n",
    "    plt.axis('off')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 28, 28)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000,)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f7cc4506978>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADVlJREFUeJzt3W+IXfWdx/HPZ2OjwRZ1zGhCGp1YpI6KTcoQg8riUgx2LcQ8iHSUkmJp+qDKFvtAzZNGQQzLtjUPlkK6iYna2hbamAiyNsiKKWhwlKGapm40zjbZxGRCirEiVDPffTAn3Wmce+7N/Xfu5Pt+Qbj3nu/58+WSz5x77+/e83NECEA+/1B1AwCqQfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyR1TjcPNnfu3BgYGOjmIYFUxsbGdOzYMTeybkvht32rpA2SZkn6j4hYX7b+wMCARkZGWjkkgBJDQ0MNr9v0y37bsyT9u6SvSrpa0rDtq5vdH4DuauU9/1JJb0fE/oj4q6RfSFrRnrYAdFor4V8g6cCUxweLZX/H9hrbI7ZHxsfHWzgcgHZqJfzTfajwqd8HR8TGiBiKiKH+/v4WDgegnVoJ/0FJC6c8/rykQ621A6BbWgn/q5KutL3I9mxJX5e0oz1tAei0pof6IuIT2/dIel6TQ32bI2JP2zoD0FEtjfNHxHOSnmtTLwC6iK/3AkkRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kFRLs/TaHpP0gaSTkj6JiKF2NAWg81oKf+GfIuJYG/YDoIt42Q8k1Wr4Q9Jvbb9me007GgLQHa2+7L8xIg7ZvkTSTtt/jIiXpq5Q/FFYI0mXXXZZi4cD0C4tnfkj4lBxe1TSNklLp1lnY0QMRcRQf39/K4cD0EZNh9/2+bY/d+q+pOWS3mxXYwA6q5WX/ZdK2mb71H5+HhH/2ZauAHRc0+GPiP2SvtTGXgB0EUN9QFKEH0iK8ANJEX4gKcIPJEX4gaTa8au+FF555ZWatQ0bNpRuu2DBgtL6nDlzSuurV68urff19TVVQ26c+YGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcb5G1Q21r5v376OHvuRRx4prV9wwQU1a8uWLWt3OzPGwMBAzdqDDz5Yum2GS85x5geSIvxAUoQfSIrwA0kRfiApwg8kRfiBpBjnb9AzzzxTszY6Olq67TXXXFNa37NnT2l99+7dpfXt27fXrD3//POl2y5atKi0/u6775bWW3HOOeX//ebPn19aP3DgQNPHLvsOgCTdf//9Te97puDMDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJ1R3nt71Z0tckHY2Ia4tlfZJ+KWlA0pikOyLiz51rs3qDg4NN1Rpx3XXXldaHh4dL6+vXr69ZGxsbK9223jj//v37S+utmD17dmm93jh/vd7Hx8dr1q666qrSbTNo5My/RdKtpy17QNILEXGlpBeKxwBmkLrhj4iXJB0/bfEKSVuL+1sl3d7mvgB0WLPv+S+NiMOSVNxe0r6WAHRDxz/ws73G9ojtkbL3YAC6q9nwH7E9X5KK26O1VoyIjRExFBFD/f39TR4OQLs1G/4dkk5dzna1pNo/KwPQk+qG3/bTkl6W9EXbB21/S9J6SbfY3ifpluIxgBmk7jh/RNQaZP5Km3tBk84777yatVbHs1v9DkMr6l3H4NixY6X166+/vmZt+fLlTfV0NuEbfkBShB9IivADSRF+ICnCDyRF+IGkuHQ3KvPhhx+W1leuXFlan5iYKK0/9thjNWtz5swp3TYDzvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBTj/KjMli1bSuvvvfdeaf3iiy8urV9++eVn2lIqnPmBpAg/kBThB5Ii/EBShB9IivADSRF+ICnG+dFR77zzTs3afffd19K+X3755dL6vHnzWtr/2Y4zP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kVXec3/ZmSV+TdDQiri2WrZP0bUnjxWprI+K5TjWJmevZZ5+tWfv4449Lt121alVp/YorrmiqJ0xq5My/RdKt0yz/cUQsLv4RfGCGqRv+iHhJ0vEu9AKgi1p5z3+P7d/b3mz7orZ1BKArmg3/TyR9QdJiSYcl/bDWirbX2B6xPTI+Pl5rNQBd1lT4I+JIRJyMiAlJP5W0tGTdjRExFBFD/f39zfYJoM2aCr/t+VMerpT0ZnvaAdAtjQz1PS3pZklzbR+U9ANJN9teLCkkjUn6Tgd7BNABdcMfEcPTLN7UgV4wA9Ubq9+2bVvN2rnnnlu67aOPPlpanzVrVmkd5fiGH5AU4QeSIvxAUoQfSIrwA0kRfiApLt2NlmzaVD7qu2vXrpq1O++8s3RbfrLbWZz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApxvlRanR0tLR+7733ltYvvPDCmrWHH364qZ7QHpz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApxvmT++ijj0rrw8PTXbn9/508ebK0ftddd9Ws8Xv9anHmB5Ii/EBShB9IivADSRF+ICnCDyRF+IGk6o7z214o6QlJ8yRNSNoYERts90n6paQBSWOS7oiIP3euVTRjYmKitH7bbbeV1t96663S+uDgYGn9oYceKq2jOo2c+T+R9P2IGJS0TNJ3bV8t6QFJL0TElZJeKB4DmCHqhj8iDkfE68X9DyTtlbRA0gpJW4vVtkq6vVNNAmi/M3rPb3tA0hJJuyVdGhGHpck/EJIuaXdzADqn4fDb/qykX0v6XkScOIPt1tgesT0yPj7eTI8AOqCh8Nv+jCaD/7OI+E2x+Ijt+UV9vqSj020bERsjYigihvr7+9vRM4A2qBt+25a0SdLeiPjRlNIOSauL+6slbW9/ewA6pZGf9N4o6RuS3rB96jrOayWtl/Qr29+S9CdJqzrTIlpx/Pjx0vqLL77Y0v6ffPLJ0npfX19L+0fn1A1/RPxOkmuUv9LedgB0C9/wA5Ii/EBShB9IivADSRF+ICnCDyTFpbvPAu+//37N2rJly1ra91NPPVVaX7JkSUv7R3U48wNJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUozznwUef/zxmrX9+/e3tO+bbrqptD55rRfMRJz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApxvlngH379pXW161b151GcFbhzA8kRfiBpAg/kBThB5Ii/EBShB9IivADSdUd57e9UNITkuZJmpC0MSI22F4n6duSxotV10bEc51qNLNdu3aV1k+cONH0vgcHB0vrc+bMaXrf6G2NfMnnE0nfj4jXbX9O0mu2dxa1H0fEv3WuPQCdUjf8EXFY0uHi/ge290pa0OnGAHTWGb3ntz0gaYmk3cWie2z/3vZm2xfV2GaN7RHbI+Pj49OtAqACDYff9mcl/VrS9yLihKSfSPqCpMWafGXww+m2i4iNETEUEUP9/f1taBlAOzQUftuf0WTwfxYRv5GkiDgSEScjYkLSTyUt7VybANqtbvg9eXnWTZL2RsSPpiyfP2W1lZLebH97ADqlkU/7b5T0DUlv2B4tlq2VNGx7saSQNCbpOx3pEC254YYbSus7d+4srTPUd/Zq5NP+30ma7uLsjOkDMxjf8AOSIvxAUoQfSIrwA0kRfiApwg8kxaW7Z4C77767pTowHc78QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5CUI6J7B7PHJf3PlEVzJR3rWgNnpld769W+JHprVjt7uzwiGrpeXlfD/6mD2yMRMVRZAyV6tbde7Uuit2ZV1Rsv+4GkCD+QVNXh31jx8cv0am+92pdEb82qpLdK3/MDqE7VZ34AFakk/LZvtf2W7bdtP1BFD7XYHrP9hu1R2yMV97LZ9lHbb05Z1md7p+19xe2006RV1Ns62/9bPHejtv+5ot4W2v4v23tt77H9L8XySp+7kr4qed66/rLf9ixJ/y3pFkkHJb0qaTgi/tDVRmqwPSZpKCIqHxO2/Y+S/iLpiYi4tlj2r5KOR8T64g/nRRFxf4/0tk7SX6qeubmYUGb+1JmlJd0u6Zuq8Lkr6esOVfC8VXHmXyrp7YjYHxF/lfQLSSsq6KPnRcRLko6ftniFpK3F/a2a/M/TdTV66wkRcTgiXi/ufyDp1MzSlT53JX1VoorwL5B0YMrjg+qtKb9D0m9tv2Z7TdXNTOPSYtr0U9OnX1JxP6erO3NzN502s3TPPHfNzHjdblWEf7rZf3ppyOHGiPiypK9K+m7x8haNaWjm5m6ZZmbpntDsjNftVkX4D0paOOXx5yUdqqCPaUXEoeL2qKRt6r3Zh4+cmiS1uD1acT9/00szN083s7R64LnrpRmvqwj/q5KutL3I9mxJX5e0o4I+PsX2+cUHMbJ9vqTl6r3Zh3dIWl3cXy1pe4W9/J1embm51szSqvi567UZryv5kk8xlPGYpFmSNkfEI11vYhq2r9Dk2V6avLLxz6vszfbTkm7W5K++jkj6gaRnJP1K0mWS/iRpVUR0/YO3Gr3drMmXrn+bufnUe+wu93aTpF2S3pA0USxeq8n315U9dyV9DauC541v+AFJ8Q0/ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJ/R8EiLFW9B5y7gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7ccbdecf98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(X_valid[0], cmap='Greys')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,  84, 185, 159, 151,  60,  36,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0, 222, 254, 254, 254, 254, 241, 198,\n",
       "        198, 198, 198, 198, 198, 198, 198, 170,  52,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,  67, 114,  72, 114, 163, 227, 254,\n",
       "        225, 254, 254, 254, 250, 229, 254, 254, 140,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  17,  66,\n",
       "         14,  67,  67,  67,  59,  21, 236, 254, 106,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,  83, 253, 209,  18,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,  22, 233, 255,  83,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0, 129, 254, 238,  44,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,  59, 249, 254,  62,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0, 133, 254, 187,   5,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   9, 205, 248,  58,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0, 126, 254, 182,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,  75, 251, 240,  57,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         19, 221, 254, 166,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   3,\n",
       "        203, 254, 219,  35,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  38,\n",
       "        254, 254,  77,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  31, 224,\n",
       "        254, 115,   1,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 133, 254,\n",
       "        254,  52,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  61, 242, 254,\n",
       "        254,  52,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 121, 254, 254,\n",
       "        219,  40,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 121, 254, 207,\n",
       "         18,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0]], dtype=uint8)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_valid[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_valid[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.reshape(60000, 784).astype('float32')\n",
    "X_valid = X_valid.reshape(10000, 784).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train /= 255\n",
    "X_valid /= 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.32941177, 0.7254902 , 0.62352943,\n",
       "       0.5921569 , 0.23529412, 0.14117648, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.87058824, 0.99607843, 0.99607843, 0.99607843, 0.99607843,\n",
       "       0.94509804, 0.7764706 , 0.7764706 , 0.7764706 , 0.7764706 ,\n",
       "       0.7764706 , 0.7764706 , 0.7764706 , 0.7764706 , 0.6666667 ,\n",
       "       0.20392157, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.2627451 , 0.44705883,\n",
       "       0.28235295, 0.44705883, 0.6392157 , 0.8901961 , 0.99607843,\n",
       "       0.88235295, 0.99607843, 0.99607843, 0.99607843, 0.98039216,\n",
       "       0.8980392 , 0.99607843, 0.99607843, 0.54901963, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.06666667, 0.25882354, 0.05490196, 0.2627451 ,\n",
       "       0.2627451 , 0.2627451 , 0.23137255, 0.08235294, 0.9254902 ,\n",
       "       0.99607843, 0.41568628, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.3254902 , 0.99215686, 0.81960785, 0.07058824,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.08627451, 0.9137255 ,\n",
       "       1.        , 0.3254902 , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.5058824 , 0.99607843, 0.93333334, 0.17254902,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.23137255, 0.9764706 ,\n",
       "       0.99607843, 0.24313726, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.52156866, 0.99607843, 0.73333335, 0.01960784,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.03529412, 0.8039216 ,\n",
       "       0.972549  , 0.22745098, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.49411765, 0.99607843, 0.7137255 , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.29411766, 0.9843137 ,\n",
       "       0.9411765 , 0.22352941, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.07450981, 0.8666667 , 0.99607843, 0.6509804 , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.01176471, 0.79607844, 0.99607843,\n",
       "       0.85882354, 0.13725491, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.14901961, 0.99607843, 0.99607843, 0.3019608 , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.12156863, 0.8784314 , 0.99607843,\n",
       "       0.4509804 , 0.00392157, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.52156866, 0.99607843, 0.99607843, 0.20392157, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.23921569, 0.9490196 , 0.99607843,\n",
       "       0.99607843, 0.20392157, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.4745098 , 0.99607843, 0.99607843, 0.85882354, 0.15686275,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.4745098 , 0.99607843,\n",
       "       0.8117647 , 0.07058824, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        ], dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_valid[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_classes = 10\n",
    "y_train = keras.utils.to_categorical(y_train, n_classes)\n",
    "y_valid = keras.utils.to_categorical(y_valid, n_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 0., 0., 0., 1., 0., 0.], dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_valid[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(64, activation='sigmoid', input_shape=(784,)))\n",
    "model.add(Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 64)                50240     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                650       \n",
      "=================================================================\n",
      "Total params: 50,890\n",
      "Trainable params: 50,890\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50176"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(64*784)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50240"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(64*784)+64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "650"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(10*64)+10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configure model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='mean_squared_error', optimizer=SGD(lr=0.01), metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "60000/60000 [==============================] - 1s 12us/step - loss: 0.0923 - acc: 0.0943 - val_loss: 0.0919 - val_acc: 0.1010\n",
      "Epoch 2/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0916 - acc: 0.0979 - val_loss: 0.0912 - val_acc: 0.1075\n",
      "Epoch 3/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0910 - acc: 0.1045 - val_loss: 0.0907 - val_acc: 0.1165\n",
      "Epoch 4/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0905 - acc: 0.1157 - val_loss: 0.0902 - val_acc: 0.1330\n",
      "Epoch 5/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0900 - acc: 0.1386 - val_loss: 0.0898 - val_acc: 0.1653\n",
      "Epoch 6/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0896 - acc: 0.1754 - val_loss: 0.0894 - val_acc: 0.2143\n",
      "Epoch 7/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0892 - acc: 0.2260 - val_loss: 0.0890 - val_acc: 0.2720\n",
      "Epoch 8/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0889 - acc: 0.2820 - val_loss: 0.0887 - val_acc: 0.3124\n",
      "Epoch 9/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0885 - acc: 0.3227 - val_loss: 0.0883 - val_acc: 0.3474\n",
      "Epoch 10/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0882 - acc: 0.3527 - val_loss: 0.0880 - val_acc: 0.3649\n",
      "Epoch 11/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0878 - acc: 0.3717 - val_loss: 0.0876 - val_acc: 0.3791\n",
      "Epoch 12/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0875 - acc: 0.3777 - val_loss: 0.0873 - val_acc: 0.3815\n",
      "Epoch 13/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0871 - acc: 0.3794 - val_loss: 0.0869 - val_acc: 0.3820\n",
      "Epoch 14/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0868 - acc: 0.3786 - val_loss: 0.0866 - val_acc: 0.3811\n",
      "Epoch 15/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0864 - acc: 0.3783 - val_loss: 0.0862 - val_acc: 0.3789\n",
      "Epoch 16/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0861 - acc: 0.3787 - val_loss: 0.0858 - val_acc: 0.3799\n",
      "Epoch 17/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0857 - acc: 0.3801 - val_loss: 0.0854 - val_acc: 0.3794\n",
      "Epoch 18/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0853 - acc: 0.3799 - val_loss: 0.0851 - val_acc: 0.3815\n",
      "Epoch 19/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0849 - acc: 0.3815 - val_loss: 0.0847 - val_acc: 0.3818\n",
      "Epoch 20/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0845 - acc: 0.3816 - val_loss: 0.0843 - val_acc: 0.3836\n",
      "Epoch 21/200\n",
      "60000/60000 [==============================] - 1s 9us/step - loss: 0.0842 - acc: 0.3845 - val_loss: 0.0839 - val_acc: 0.3861\n",
      "Epoch 22/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0838 - acc: 0.3867 - val_loss: 0.0835 - val_acc: 0.3882\n",
      "Epoch 23/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0833 - acc: 0.3893 - val_loss: 0.0830 - val_acc: 0.3917\n",
      "Epoch 24/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0829 - acc: 0.3913 - val_loss: 0.0826 - val_acc: 0.3956\n",
      "Epoch 25/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0825 - acc: 0.3951 - val_loss: 0.0822 - val_acc: 0.4001\n",
      "Epoch 26/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0821 - acc: 0.3993 - val_loss: 0.0817 - val_acc: 0.4034\n",
      "Epoch 27/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0816 - acc: 0.4021 - val_loss: 0.0813 - val_acc: 0.4075\n",
      "Epoch 28/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0812 - acc: 0.4078 - val_loss: 0.0808 - val_acc: 0.4118\n",
      "Epoch 29/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0808 - acc: 0.4130 - val_loss: 0.0804 - val_acc: 0.4172\n",
      "Epoch 30/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0803 - acc: 0.4172 - val_loss: 0.0799 - val_acc: 0.4219\n",
      "Epoch 31/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0798 - acc: 0.4243 - val_loss: 0.0795 - val_acc: 0.4271\n",
      "Epoch 32/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0794 - acc: 0.4297 - val_loss: 0.0790 - val_acc: 0.4309\n",
      "Epoch 33/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0789 - acc: 0.4348 - val_loss: 0.0785 - val_acc: 0.4365\n",
      "Epoch 34/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0784 - acc: 0.4402 - val_loss: 0.0780 - val_acc: 0.4418\n",
      "Epoch 35/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0780 - acc: 0.4466 - val_loss: 0.0776 - val_acc: 0.4464\n",
      "Epoch 36/200\n",
      "60000/60000 [==============================] - 1s 9us/step - loss: 0.0775 - acc: 0.4527 - val_loss: 0.0771 - val_acc: 0.4521\n",
      "Epoch 37/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0770 - acc: 0.4583 - val_loss: 0.0766 - val_acc: 0.4592\n",
      "Epoch 38/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0765 - acc: 0.4657 - val_loss: 0.0761 - val_acc: 0.4657\n",
      "Epoch 39/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0761 - acc: 0.4716 - val_loss: 0.0756 - val_acc: 0.4722\n",
      "Epoch 40/200\n",
      "60000/60000 [==============================] - 1s 9us/step - loss: 0.0756 - acc: 0.4778 - val_loss: 0.0751 - val_acc: 0.4782\n",
      "Epoch 41/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0751 - acc: 0.4840 - val_loss: 0.0746 - val_acc: 0.4849\n",
      "Epoch 42/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0746 - acc: 0.4901 - val_loss: 0.0741 - val_acc: 0.4909\n",
      "Epoch 43/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0741 - acc: 0.4960 - val_loss: 0.0736 - val_acc: 0.4966\n",
      "Epoch 44/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0736 - acc: 0.5030 - val_loss: 0.0732 - val_acc: 0.5034\n",
      "Epoch 45/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0731 - acc: 0.5084 - val_loss: 0.0727 - val_acc: 0.5094\n",
      "Epoch 46/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0727 - acc: 0.5138 - val_loss: 0.0722 - val_acc: 0.5155\n",
      "Epoch 47/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0722 - acc: 0.5195 - val_loss: 0.0717 - val_acc: 0.5193\n",
      "Epoch 48/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0717 - acc: 0.5243 - val_loss: 0.0712 - val_acc: 0.5251\n",
      "Epoch 49/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0712 - acc: 0.5292 - val_loss: 0.0707 - val_acc: 0.5310\n",
      "Epoch 50/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0707 - acc: 0.5343 - val_loss: 0.0702 - val_acc: 0.5364\n",
      "Epoch 51/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0702 - acc: 0.5395 - val_loss: 0.0697 - val_acc: 0.5420\n",
      "Epoch 52/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0698 - acc: 0.5440 - val_loss: 0.0693 - val_acc: 0.5467\n",
      "Epoch 53/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0693 - acc: 0.5488 - val_loss: 0.0688 - val_acc: 0.5521\n",
      "Epoch 54/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0688 - acc: 0.5528 - val_loss: 0.0683 - val_acc: 0.5568\n",
      "Epoch 55/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0683 - acc: 0.5574 - val_loss: 0.0678 - val_acc: 0.5616\n",
      "Epoch 56/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0679 - acc: 0.5615 - val_loss: 0.0673 - val_acc: 0.5668\n",
      "Epoch 57/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0674 - acc: 0.5663 - val_loss: 0.0669 - val_acc: 0.5706\n",
      "Epoch 58/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0669 - acc: 0.5705 - val_loss: 0.0664 - val_acc: 0.5742\n",
      "Epoch 59/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0665 - acc: 0.5743 - val_loss: 0.0659 - val_acc: 0.5778\n",
      "Epoch 60/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0660 - acc: 0.5790 - val_loss: 0.0655 - val_acc: 0.5824\n",
      "Epoch 61/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0655 - acc: 0.5830 - val_loss: 0.0650 - val_acc: 0.5888\n",
      "Epoch 62/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0651 - acc: 0.5870 - val_loss: 0.0645 - val_acc: 0.5937\n",
      "Epoch 63/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0646 - acc: 0.5910 - val_loss: 0.0641 - val_acc: 0.5967\n",
      "Epoch 64/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0642 - acc: 0.5957 - val_loss: 0.0636 - val_acc: 0.6013\n",
      "Epoch 65/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0637 - acc: 0.5997 - val_loss: 0.0632 - val_acc: 0.6067\n",
      "Epoch 66/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0633 - acc: 0.6039 - val_loss: 0.0627 - val_acc: 0.6120\n",
      "Epoch 67/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0628 - acc: 0.6085 - val_loss: 0.0623 - val_acc: 0.6174\n",
      "Epoch 68/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0624 - acc: 0.6129 - val_loss: 0.0618 - val_acc: 0.6216\n",
      "Epoch 69/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0619 - acc: 0.6171 - val_loss: 0.0614 - val_acc: 0.6249\n",
      "Epoch 70/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0615 - acc: 0.6213 - val_loss: 0.0609 - val_acc: 0.6301\n",
      "Epoch 71/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0610 - acc: 0.6256 - val_loss: 0.0605 - val_acc: 0.6346\n",
      "Epoch 72/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0606 - acc: 0.6294 - val_loss: 0.0600 - val_acc: 0.6392\n",
      "Epoch 73/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0602 - acc: 0.6340 - val_loss: 0.0596 - val_acc: 0.6437\n",
      "Epoch 74/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0598 - acc: 0.6378 - val_loss: 0.0592 - val_acc: 0.6475\n",
      "Epoch 75/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0593 - acc: 0.6416 - val_loss: 0.0587 - val_acc: 0.6505\n",
      "Epoch 76/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0589 - acc: 0.6460 - val_loss: 0.0583 - val_acc: 0.6544\n",
      "Epoch 77/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0585 - acc: 0.6499 - val_loss: 0.0579 - val_acc: 0.6592\n",
      "Epoch 78/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0581 - acc: 0.6537 - val_loss: 0.0575 - val_acc: 0.6635\n",
      "Epoch 79/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0577 - acc: 0.6572 - val_loss: 0.0570 - val_acc: 0.6664\n",
      "Epoch 80/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0572 - acc: 0.6612 - val_loss: 0.0566 - val_acc: 0.6698\n",
      "Epoch 81/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0568 - acc: 0.6647 - val_loss: 0.0562 - val_acc: 0.6731\n",
      "Epoch 82/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0564 - acc: 0.6683 - val_loss: 0.0558 - val_acc: 0.6762\n",
      "Epoch 83/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0560 - acc: 0.6714 - val_loss: 0.0554 - val_acc: 0.6795\n",
      "Epoch 84/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0556 - acc: 0.6748 - val_loss: 0.0550 - val_acc: 0.6833\n",
      "Epoch 85/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0552 - acc: 0.6794 - val_loss: 0.0546 - val_acc: 0.6868\n",
      "Epoch 86/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0548 - acc: 0.6826 - val_loss: 0.0542 - val_acc: 0.6903\n",
      "Epoch 87/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0545 - acc: 0.6855 - val_loss: 0.0538 - val_acc: 0.6942\n",
      "Epoch 88/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0541 - acc: 0.6890 - val_loss: 0.0534 - val_acc: 0.6986\n",
      "Epoch 89/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0537 - acc: 0.6922 - val_loss: 0.0530 - val_acc: 0.7018\n",
      "Epoch 90/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0533 - acc: 0.6948 - val_loss: 0.0526 - val_acc: 0.7049\n",
      "Epoch 91/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0529 - acc: 0.6976 - val_loss: 0.0522 - val_acc: 0.7083\n",
      "Epoch 92/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0526 - acc: 0.7001 - val_loss: 0.0519 - val_acc: 0.7115\n",
      "Epoch 93/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0522 - acc: 0.7031 - val_loss: 0.0515 - val_acc: 0.7142\n",
      "Epoch 94/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0518 - acc: 0.7065 - val_loss: 0.0511 - val_acc: 0.7174\n",
      "Epoch 95/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0515 - acc: 0.7095 - val_loss: 0.0507 - val_acc: 0.7206\n",
      "Epoch 96/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0511 - acc: 0.7121 - val_loss: 0.0504 - val_acc: 0.7227\n",
      "Epoch 97/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0507 - acc: 0.7151 - val_loss: 0.0500 - val_acc: 0.7259\n",
      "Epoch 98/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0504 - acc: 0.7179 - val_loss: 0.0497 - val_acc: 0.7287\n",
      "Epoch 99/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0500 - acc: 0.7209 - val_loss: 0.0493 - val_acc: 0.7310\n",
      "Epoch 100/200\n",
      "60000/60000 [==============================] - 1s 9us/step - loss: 0.0497 - acc: 0.7236 - val_loss: 0.0489 - val_acc: 0.7340\n",
      "Epoch 101/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0493 - acc: 0.7262 - val_loss: 0.0486 - val_acc: 0.7370\n",
      "Epoch 102/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0490 - acc: 0.7286 - val_loss: 0.0483 - val_acc: 0.7397\n",
      "Epoch 103/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0487 - acc: 0.7313 - val_loss: 0.0479 - val_acc: 0.7418\n",
      "Epoch 104/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0483 - acc: 0.7334 - val_loss: 0.0476 - val_acc: 0.7445\n",
      "Epoch 105/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0480 - acc: 0.7360 - val_loss: 0.0472 - val_acc: 0.7464\n",
      "Epoch 106/200\n",
      "60000/60000 [==============================] - 1s 9us/step - loss: 0.0477 - acc: 0.7385 - val_loss: 0.0469 - val_acc: 0.7499\n",
      "Epoch 107/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0474 - acc: 0.7408 - val_loss: 0.0466 - val_acc: 0.7514\n",
      "Epoch 108/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0471 - acc: 0.7435 - val_loss: 0.0463 - val_acc: 0.7533\n",
      "Epoch 109/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0468 - acc: 0.7451 - val_loss: 0.0460 - val_acc: 0.7564\n",
      "Epoch 110/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0465 - acc: 0.7468 - val_loss: 0.0457 - val_acc: 0.7589\n",
      "Epoch 111/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0462 - acc: 0.7494 - val_loss: 0.0453 - val_acc: 0.7603\n",
      "Epoch 112/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0459 - acc: 0.7510 - val_loss: 0.0450 - val_acc: 0.7615\n",
      "Epoch 113/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0456 - acc: 0.7531 - val_loss: 0.0447 - val_acc: 0.7631\n",
      "Epoch 114/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0453 - acc: 0.7549 - val_loss: 0.0444 - val_acc: 0.7649\n",
      "Epoch 115/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0450 - acc: 0.7562 - val_loss: 0.0442 - val_acc: 0.7665\n",
      "Epoch 116/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0447 - acc: 0.7582 - val_loss: 0.0439 - val_acc: 0.7684\n",
      "Epoch 117/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0444 - acc: 0.7595 - val_loss: 0.0436 - val_acc: 0.7693\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 118/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0442 - acc: 0.7608 - val_loss: 0.0433 - val_acc: 0.7708\n",
      "Epoch 119/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0439 - acc: 0.7624 - val_loss: 0.0430 - val_acc: 0.7716\n",
      "Epoch 120/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0436 - acc: 0.7636 - val_loss: 0.0428 - val_acc: 0.7732\n",
      "Epoch 121/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0434 - acc: 0.7651 - val_loss: 0.0425 - val_acc: 0.7743\n",
      "Epoch 122/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0431 - acc: 0.7665 - val_loss: 0.0422 - val_acc: 0.7758\n",
      "Epoch 123/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0428 - acc: 0.7677 - val_loss: 0.0420 - val_acc: 0.7777\n",
      "Epoch 124/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0426 - acc: 0.7685 - val_loss: 0.0417 - val_acc: 0.7791\n",
      "Epoch 125/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0423 - acc: 0.7694 - val_loss: 0.0415 - val_acc: 0.7803\n",
      "Epoch 126/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0421 - acc: 0.7707 - val_loss: 0.0412 - val_acc: 0.7810\n",
      "Epoch 127/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0419 - acc: 0.7715 - val_loss: 0.0410 - val_acc: 0.7815\n",
      "Epoch 128/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0416 - acc: 0.7723 - val_loss: 0.0407 - val_acc: 0.7829\n",
      "Epoch 129/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0414 - acc: 0.7729 - val_loss: 0.0405 - val_acc: 0.7832\n",
      "Epoch 130/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0412 - acc: 0.7737 - val_loss: 0.0403 - val_acc: 0.7843\n",
      "Epoch 131/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0409 - acc: 0.7747 - val_loss: 0.0400 - val_acc: 0.7850\n",
      "Epoch 132/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0407 - acc: 0.7753 - val_loss: 0.0398 - val_acc: 0.7863\n",
      "Epoch 133/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0405 - acc: 0.7760 - val_loss: 0.0396 - val_acc: 0.7865\n",
      "Epoch 134/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0403 - acc: 0.7767 - val_loss: 0.0394 - val_acc: 0.7875\n",
      "Epoch 135/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0400 - acc: 0.7774 - val_loss: 0.0391 - val_acc: 0.7881\n",
      "Epoch 136/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0398 - acc: 0.7780 - val_loss: 0.0389 - val_acc: 0.7890\n",
      "Epoch 137/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0396 - acc: 0.7788 - val_loss: 0.0387 - val_acc: 0.7897\n",
      "Epoch 138/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0394 - acc: 0.7793 - val_loss: 0.0385 - val_acc: 0.7904\n",
      "Epoch 139/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0392 - acc: 0.7798 - val_loss: 0.0383 - val_acc: 0.7908\n",
      "Epoch 140/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0390 - acc: 0.7806 - val_loss: 0.0381 - val_acc: 0.7920\n",
      "Epoch 141/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0388 - acc: 0.7813 - val_loss: 0.0379 - val_acc: 0.7923\n",
      "Epoch 142/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0386 - acc: 0.7818 - val_loss: 0.0377 - val_acc: 0.7929\n",
      "Epoch 143/200\n",
      "60000/60000 [==============================] - 1s 11us/step - loss: 0.0384 - acc: 0.7824 - val_loss: 0.0375 - val_acc: 0.7936\n",
      "Epoch 144/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0382 - acc: 0.7828 - val_loss: 0.0373 - val_acc: 0.7942\n",
      "Epoch 145/200\n",
      "60000/60000 [==============================] - 1s 9us/step - loss: 0.0380 - acc: 0.7836 - val_loss: 0.0371 - val_acc: 0.7944\n",
      "Epoch 146/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0378 - acc: 0.7841 - val_loss: 0.0369 - val_acc: 0.7946\n",
      "Epoch 147/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0377 - acc: 0.7847 - val_loss: 0.0367 - val_acc: 0.7948\n",
      "Epoch 148/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0375 - acc: 0.7854 - val_loss: 0.0365 - val_acc: 0.7954\n",
      "Epoch 149/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0373 - acc: 0.7859 - val_loss: 0.0363 - val_acc: 0.7960\n",
      "Epoch 150/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0371 - acc: 0.7865 - val_loss: 0.0362 - val_acc: 0.7966\n",
      "Epoch 151/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0369 - acc: 0.7868 - val_loss: 0.0360 - val_acc: 0.7975\n",
      "Epoch 152/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0367 - acc: 0.7872 - val_loss: 0.0358 - val_acc: 0.7982\n",
      "Epoch 153/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0366 - acc: 0.7879 - val_loss: 0.0356 - val_acc: 0.7990\n",
      "Epoch 154/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0364 - acc: 0.7887 - val_loss: 0.0354 - val_acc: 0.7995\n",
      "Epoch 155/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0362 - acc: 0.7894 - val_loss: 0.0353 - val_acc: 0.8004\n",
      "Epoch 156/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0360 - acc: 0.7900 - val_loss: 0.0351 - val_acc: 0.8006\n",
      "Epoch 157/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0359 - acc: 0.7906 - val_loss: 0.0349 - val_acc: 0.8019\n",
      "Epoch 158/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0357 - acc: 0.7913 - val_loss: 0.0347 - val_acc: 0.8027\n",
      "Epoch 159/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0355 - acc: 0.7921 - val_loss: 0.0346 - val_acc: 0.8035\n",
      "Epoch 160/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0354 - acc: 0.7929 - val_loss: 0.0344 - val_acc: 0.8044\n",
      "Epoch 161/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0352 - acc: 0.7937 - val_loss: 0.0342 - val_acc: 0.8057\n",
      "Epoch 162/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0350 - acc: 0.7947 - val_loss: 0.0341 - val_acc: 0.8070\n",
      "Epoch 163/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0349 - acc: 0.7959 - val_loss: 0.0339 - val_acc: 0.8083\n",
      "Epoch 164/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0347 - acc: 0.7970 - val_loss: 0.0337 - val_acc: 0.8094\n",
      "Epoch 165/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0345 - acc: 0.7982 - val_loss: 0.0336 - val_acc: 0.8106\n",
      "Epoch 166/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0344 - acc: 0.7993 - val_loss: 0.0334 - val_acc: 0.8121\n",
      "Epoch 167/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0342 - acc: 0.8008 - val_loss: 0.0333 - val_acc: 0.8135\n",
      "Epoch 168/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0341 - acc: 0.8021 - val_loss: 0.0331 - val_acc: 0.8149\n",
      "Epoch 169/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0339 - acc: 0.8035 - val_loss: 0.0329 - val_acc: 0.8168\n",
      "Epoch 170/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0337 - acc: 0.8050 - val_loss: 0.0328 - val_acc: 0.8187\n",
      "Epoch 171/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0336 - acc: 0.8066 - val_loss: 0.0326 - val_acc: 0.8199\n",
      "Epoch 172/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0334 - acc: 0.8079 - val_loss: 0.0325 - val_acc: 0.8205\n",
      "Epoch 173/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0333 - acc: 0.8092 - val_loss: 0.0323 - val_acc: 0.8221\n",
      "Epoch 174/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0331 - acc: 0.8111 - val_loss: 0.0322 - val_acc: 0.8240\n",
      "Epoch 175/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0330 - acc: 0.8127 - val_loss: 0.0320 - val_acc: 0.8247\n",
      "Epoch 176/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0328 - acc: 0.8141 - val_loss: 0.0319 - val_acc: 0.8269\n",
      "Epoch 177/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0327 - acc: 0.8160 - val_loss: 0.0317 - val_acc: 0.8279\n",
      "Epoch 178/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0325 - acc: 0.8174 - val_loss: 0.0316 - val_acc: 0.8292\n",
      "Epoch 179/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0324 - acc: 0.8185 - val_loss: 0.0314 - val_acc: 0.8303\n",
      "Epoch 180/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0323 - acc: 0.8201 - val_loss: 0.0313 - val_acc: 0.8322\n",
      "Epoch 181/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0321 - acc: 0.8217 - val_loss: 0.0311 - val_acc: 0.8341\n",
      "Epoch 182/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0320 - acc: 0.8230 - val_loss: 0.0310 - val_acc: 0.8358\n",
      "Epoch 183/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0318 - acc: 0.8245 - val_loss: 0.0309 - val_acc: 0.8376\n",
      "Epoch 184/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0317 - acc: 0.8256 - val_loss: 0.0307 - val_acc: 0.8382\n",
      "Epoch 185/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0316 - acc: 0.8275 - val_loss: 0.0306 - val_acc: 0.8396\n",
      "Epoch 186/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0314 - acc: 0.8287 - val_loss: 0.0304 - val_acc: 0.8407\n",
      "Epoch 187/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0313 - acc: 0.8301 - val_loss: 0.0303 - val_acc: 0.8427\n",
      "Epoch 188/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0311 - acc: 0.8310 - val_loss: 0.0302 - val_acc: 0.8436\n",
      "Epoch 189/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0310 - acc: 0.8323 - val_loss: 0.0300 - val_acc: 0.8454\n",
      "Epoch 190/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0309 - acc: 0.8338 - val_loss: 0.0299 - val_acc: 0.8469\n",
      "Epoch 191/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0307 - acc: 0.8348 - val_loss: 0.0298 - val_acc: 0.8483\n",
      "Epoch 192/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0306 - acc: 0.8360 - val_loss: 0.0296 - val_acc: 0.8492\n",
      "Epoch 193/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0305 - acc: 0.8371 - val_loss: 0.0295 - val_acc: 0.8498\n",
      "Epoch 194/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0304 - acc: 0.8383 - val_loss: 0.0294 - val_acc: 0.8509\n",
      "Epoch 195/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0302 - acc: 0.8396 - val_loss: 0.0292 - val_acc: 0.8518\n",
      "Epoch 196/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0301 - acc: 0.8405 - val_loss: 0.0291 - val_acc: 0.8525\n",
      "Epoch 197/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0300 - acc: 0.8414 - val_loss: 0.0290 - val_acc: 0.8532\n",
      "Epoch 198/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0299 - acc: 0.8425 - val_loss: 0.0289 - val_acc: 0.8544\n",
      "Epoch 199/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0297 - acc: 0.8437 - val_loss: 0.0287 - val_acc: 0.8548\n",
      "Epoch 200/200\n",
      "60000/60000 [==============================] - 1s 10us/step - loss: 0.0296 - acc: 0.8446 - val_loss: 0.0286 - val_acc: 0.8560\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f7ccd479b38>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, batch_size=128, epochs=200, verbose=1, validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 11us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.02861827013194561, 0.856]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(X_valid, y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
